#爬取永康房地产网出售房源信息
import requests
from lxml import etree
import random
import re
from datetime import datetime
from time import time,sleep
from openpyxl import Workbook
from threading import Thread
from queue import Queue
from tkinter import Tk,Label

class YkHouse:
    def request(self,headers,url,p):  #请求链接
        response = requests.get(url,headers)
        response.encoding = 'gbk'
        text = response.text
        xml = etree.HTML(text)
        self.jiexi_href(xml,p)
        
    def jiexi_href(self,xml,p):  #解析各房源链接
        href_list = xml.xpath('//div[@class="houselistleft"]//span[2]/p[1]/a/@href')
        hf = []  #链接
        for i in range(0,len(href_list)):
            href = 'http://www.ykfdcw.com/house/'+href_list[i]
            hf.append(href)
        self.jiexi_detail(hf,p)
        
    def jiexi_detail(self,hf,p):
        #在循环前先新建工作表，写入首行类目
        sheetname = '第'+str(p+1)+'页'
        sheet = book.create_sheet(sheetname)
        sheet.append(['房产名称','总价','均价','面积','户型','类型','朝向','建造时间',
                      '装修','土地','车库','备注','浏览人次','上架时间'])
        k = 0
        for i in hf:
            k += 1
            response = requests.get(i,headers=headers)
            response.encoding='gbk'
            xml = etree.HTML(response.text)
            
            info = xml.xpath('//div[@class="hsopenitem"]/ul//text()')
            title = xml.xpath('//div[@class="hsopenitem1"]/ul/li[2]/text()')
            inf = []
            for i in info:
                if ' ' in i:
                    continue
                else:inf.append(i)
            liulan = xml.xpath('//div[@class="hsopentitle"]/ul[3]/text()')
            
            beizhu = xml.xpath('//div[@id="details"]/div[6]//text()')  #备注
            inf = title[0] + ''.join(inf)
            #print(k)
            self.clean(inf,liulan,beizhu,k,sheet)
        
    def clean(self,inf,liulan,beizhu,k,sheet):
        tit = re.findall('(.*?)出售',inf)[0]
        zongjia = re.findall('报价：([0-9].*?)万',inf)
        if zongjia == []:
            zj = '面议'
        else:
            zj = zongjia[0]
        junjia = re.findall(r'\s([0-9].*?)元/㎡',inf)
        if junjia == []:
            jj = '无'
        else:
            jj = junjia[0]
        s = re.findall('面积：([0-9].*?)㎡',inf)[0]
        huxing = re.findall('户型：(.*?)房产',inf)[0]
        leixing = re.findall('类型：(.*?)朝向',inf)[0]
        chaoxiang = re.findall('朝向：(.*?)建造',inf)[0]
        build_time = re.findall('时间：(.*?)装修',inf)[0]
        zhuangxiu = re.findall(r'装修：(.*?)土地',inf)[0]
        tudi = re.findall('性质：(.*?)证件',inf)[0]
        cheku = re.findall('车库：(.*?)所属',inf)
        if cheku == []:
            ck = ''
        else:
            ck = cheku[0]
        renci = re.findall('：([0-9]*?)人次',liulan[0])[0]
        show_time = re.findall('201.*',liulan[0])[0]
        bz = re.sub(' ','',''.join(beizhu))

        self.write_to_excel(tit,zj,jj,s,huxing,leixing,chaoxiang,sheet,
                            build_time,zhuangxiu,tudi,ck,renci,show_time,bz,k)

    def write_to_excel(self,tit,zj,jj,s,huxing,leixing,chaoxiang,sheet,
                       build_time,zhuangxiu,tudi,ck,renci,show_time,bz,k):
        sheet.append([tit,zj,jj,s,huxing,leixing,chaoxiang,build_time,zhuangxiu,
                      tudi,ck,bz,renci,show_time])

class ThreadPool:

    def __init__(self,n):
        self.queue = Queue()
        for i in range(n):      # 创建线程
            Thread(target=self.worker,daemon=True).start()

    def worker(self):
        while True:
            func,args,kwargs = self.queue.get()
            func(*args,**kwargs)
            self.queue.task_done()

    def apply_async(self,target,args = (),kwargs = {}):
        self.queue.put((target,args,kwargs))    # 入队

    def join(self):
        self.queue.join()

def f(page_start, page_end, name, k):
    global page_count
    count = 0
    for p in range(page_start, page_end):
        if p == 0:
            url = url0
        else:
            url = url0 + '?curpage=' + str(p + 1) + '&xxlb=4'
        repeat(headers, url, p)
        book.save(title)
        # print(f'已保存第{p+1}页')
        page_count += 1
        count += 1
        # rate = count / pagenum * 100
        # jindu = '█' * round(rate / 5) + '__' * (20 - round(rate / 5))
        # print('\r' + jindu + f'进度={round(rate,2)}%', end='')
        # label0 = Label(root, text = name, anchor = 'w')
        # label0.place(x=20, y=10 + (k - 1) * 30, width=30, height=30)
        # label1 = Label(root, text = jindu + ' ' + f'进度={round(rate,2)}%', anchor = 'w')
        # label1.place(x=50, y=10 + (k - 1) * 30, width=280, height=30)

def repeat(headers, url, p):
    try:
        yh.request(headers, url, p)
    except:
        print('网站炸了，这页重爬')
        repeat(headers, url, p)

def time_count():
    while page_count < pagenum:
        t = time()
        dt = t - t1
        # label = Label(root, text=' '*20)
        # label.place(x=20, y=10 + 7 * 30, width=50, height=30)
        # label = Label(root, text = f'运行时间={dt//60}分{round(dt%60,2)}秒', anchor = 'w')
        # label.place(x=70, y=10 + 7 * 30, width=150, height=30)
        # sleep(0.1)
        if page_count == 0:
            continue
        else:
            T = (pagenum-page_count)/(page_count/dt)
            rate = page_count / pagenum * 100
            print('\r'+'█' * round(rate / 5) + '__' * (20 - round(rate / 5)) +
                  f'进度={round(rate,2)}%运行时间={dt//60}分{round(dt%60,2)}秒,预计剩余时间={T//60}分{round(T%60,2)}秒',end='')
if __name__ == "__main__":
    user_agent_list = [ \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", \
            "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \
            "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", \
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", \
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
    ]
    ua = random.choice(user_agent_list)
    headers = {'User-Agent': ua,"Referer":'http://www.ykfdcw.com/'}

    yh = YkHouse()  #实例化
    #先获取最大页数
    url0 = 'http://www.ykfdcw.com/house/hscs.asp'
    response = requests.get(url0,headers)
    response.encoding = 'gbk'
    text = response.text
    xml = etree.HTML(text)
    lastpage = xml.xpath('//div[@id="page"]//text()')  #最大页数
    pagenum = eval(lastpage[-1][-5:-2])
    print('最大页数:',pagenum)
    #新建Excel
    book = Workbook()
    now_time = str(datetime.now())[0:10]
    title = now_time + '房产出售信息(房地产网).xlsx'
    t1 = time()  #开始计时
    print('多线程开始')
    tp = ThreadPool(7)
    # root = Tk()  # 创建一个实例对象
    # root.title("进度条显示")  # 标题
    # root.minsize(350, 270)  # 窗口大小（宽×高）
    page_count = 0
    tp.apply_async(f, args=(0, 50, 'th1', 1))
    tp.apply_async(f, args=(50, 100, 'th2', 2))
    tp.apply_async(f, args=(100, 150, 'th3', 3))
    tp.apply_async(f, args=(150, 200, 'th4', 4))
    tp.apply_async(f, args=(200, 250, 'th5', 5))
    tp.apply_async(f, args=(250, 300, 'th6', 6))
    tp.apply_async(f, args=(300, pagenum, 'th7', 7))
    th = Thread(target=time_count, daemon=True)
    th.start()
    tp.join()
    if 'Sheet' in book.sheetnames:
        book.remove(book['Sheet'])
    for i in book.sheetnames:
        if i[-1] != '页':
            book.remove(book[i[:-1]])
            book[i].title = i[:-1]
    # print(book.sheetnames)
    try:
        book.save(title)
    except IndexError:
        print('继续跑')
    # root.mainloop()
    # print(f'爬取总耗时={(t2-t1)//60}分{(t2-t1)%60}秒')